package edu.uci.ics.crawler4j.examples.douban;

import edu.uci.ics.crawler4j.crawler.CrawlConfig;
import edu.uci.ics.crawler4j.crawler.CrawlController;
import edu.uci.ics.crawler4j.fetcher.PageFetcher;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtConfig;
import edu.uci.ics.crawler4j.robotstxt.RobotstxtServer;

/**
 * Created by liuyaowen on 2016/8/25.
 */
public class DoubanlController {

    public static void main(String[] args) throws Exception {
        String crawlStorageFolder = "XXXXX";//项目目录
        int numberOfCrawlers = 1;//多少个线程

        CrawlConfig config = new CrawlConfig();
        config.setCrawlStorageFolder(crawlStorageFolder);

        config.setPolitenessDelay(1000);//延迟时间
        config.setMaxDepthOfCrawling(0);//最大深度

        config.setMaxPagesToFetch(1000);//设置最大页面获取

        config.setIncludeBinaryContentInCrawling(false);//设置在爬行包含二进制内容

        config.setResumableCrawling(false);//设置可恢复的爬行

        PageFetcher pageFetcher = new PageFetcher(config);
        RobotstxtConfig robotstxtConfig = new RobotstxtConfig();
        RobotstxtServer robotstxtServer = new RobotstxtServer(robotstxtConfig, pageFetcher);
        CrawlController controller = new CrawlController(config, pageFetcher, robotstxtServer);

        controller.addSeed("https://www.douban.com/group/topic/90118293/");

        controller.start(DoubanCrawler.class, numberOfCrawlers);
    }
}
